import crawler.WebCrawler;
import preprocessing.ChinesePreprocessor;
import preprocessing.EnglishPreprocessor;
import preprocessing.Preprocessor;
import utils.FileStorageUtil;

import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.stream.Stream;

public class Main {
    public static void main(String[] args) {
        try {
            // 初始化目录结构
            initDirectories();

            // 爬取文档
            WebCrawler crawler = new WebCrawler();
            crawler.startCrawling();

            // 预处理文档
            processDocuments();

        } catch (Exception e) {
            e.printStackTrace();
        }
    }

    private static void initDirectories() throws IOException {
        // 原始文档目录
        FileStorageUtil.ensureDirectoryExists("data/raw/english");
        FileStorageUtil.ensureDirectoryExists("data/raw/chinese");

        // 处理后的文档目录
        FileStorageUtil.ensureDirectoryExists("data/processed/english");
        FileStorageUtil.ensureDirectoryExists("data/processed/chinese");
    }

    private static void processDocuments() throws IOException {
        // 处理英文文档
        processLanguageDocuments("english", new EnglishPreprocessor(), "E");

        // 处理中文文档
        processLanguageDocuments("chinese", new ChinesePreprocessor(), "C");
    }

    private static void processLanguageDocuments(String language, Preprocessor preprocessor, String suffix)
            throws IOException {
        String rawDir = Paths.get("data", "raw", language).toString();
        String processedDir = Paths.get("data", "processed", language).toString();

        try (Stream<Path> paths = Files.list(Paths.get(rawDir))) {
            paths.forEach(path -> {
                try {
                    File inputFile = path.toFile();
                    String outputFilename = inputFile.getName()
                            .replace(".html", suffix + ".txt")
                            .replace(".htm", suffix + ".txt");

                    File outputFile = new File(processedDir, outputFilename);
                    preprocessor.preprocessFile(inputFile, outputFile);

                    System.out.println("Processed: " + outputFile.getPath());
                } catch (IOException e) {
                    System.err.println("Error processing file: " + path);
                }
            });
        }
    }
}